import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import okhttp3.*;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by lkkjzx on 2017/4/25.
 */
public class SpiderProcessor implements PageProcessor {

    OkHttpClient client = null;
    Pattern partyIdPattern = Pattern.compile("party/\\w{1,10}\\.html");
    private String location=null;

    private Map<String, Object> tempMap = new HashMap<>();

    public SpiderProcessor(String location) {
        synchronized (this) {
            if (client == null) {
                client = new OkHttpClient();
            }
            this.location=location;
        }
    }
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
    private static String partyListReg = "http://www.hdb.com/party/.*";
    private static String ListReg = "http://www.hdb.com/\\w*/0-0-2-0-\\d{1,3}";

    @Override
    public void process(Page page) {
        if (page.getUrl().regex(partyListReg).match()) {

            //活动id
            String partyId = getPartyId(page.getUrl().toString());
            page.putField("party_id", partyId);
            //主题
            Selectable head = page.getHtml().xpath("//div[@class='content-body_head_r']");

            page.putField("title", head.xpath("//h1[@id='dt_title']/text()").toString());
            page.putField("url", page.getUrl().toString());
            page.putField("banner",page.getHtml().xpath("//div[@class='content-body_head_l']/img/@src").toString());
            page.putField("location", this.location);

            Selectable user = page.getHtml().xpath("//div[contains(@class,'detail_user')]");
            //发布人
            page.putField("author", user.xpath("//a[@id='subinfo_name']/text()").toString());
            //发布人详情url
            String authorUrl=user.xpath("//a[@id='subinfo_name']/@href").toString();
            page.putField("author_url",authorUrl);
            String authorId=getAuthorId(authorUrl);
            page.putField("author_id",authorId);
            page.putField("author_info",getAuthorInfo(authorId,partyId));

//            //发布时间
            page.putField("publish_day", user.xpath("//p[@class='fbTime']/text()").toString());
//            //分享数据
//            page.putField("share_info", user.xpath("//a[@id='info_share']/text()").toString());

            //报名时间
            page.putField("join_time", head.xpath("//div[@class='detail_Time_t']/p/text()").toString());
//            //现报名人数
//            page.putField("join_count", head.xpath("//div[@id='detail_Joinnum']").toString());
            //限制报名人数
//            page.putField("limit_count", user.xpath("(//div[@id='detail_Joinnum']//span)[2]").toString());
            //有几种票,从javascript 中正则匹配

            String payItemString = page.getHtml().regex("_payItemListJson:\'.*?\'").toString();
            payItemString = payItemString.substring(18, payItemString.length() - 1);
            //获取浏览次数和分享次数
            JSONObject hitsAndShareJsonObj = JSON.parseObject(getHitAndShareString(partyId));
            page.putField("hits", hitsAndShareJsonObj.getString("hits"));
            page.putField("share", hitsAndShareJsonObj.getString("share"));

//            System.out.println(payItemString);
//            JSONArray payJSONArray = new JSONArray();
//            List<Selectable> ticketNode = page.getHtml().xpath("//ul[@class='ticket tc_c_feiLi']/li").nodes();
//            for (int i = 0; i < ticketNode.size(); i++) {
//                JSONObject payJsonObject = new JSONObject();
//                Selectable temp = ticketNode.get(i);
//                payJsonObject.put("money", temp.xpath("//span[@class='ticket_money']/text()").toString());
//                payJsonObject.put("des", temp.xpath("//div[@class='ticket_des']/text()").toString());
////                payJsonObject.put("",temp.xpath("//div[@class='ticket_des']/text()").toString());
//                System.out.println(payJsonObject.toJSONString());
//                payJSONArray.add(payJsonObject);
//            }
            page.putField("ticket_list", payItemString);

            //参加的用户
            page.putField("join_user_list", partyId == null ? new JSONArray() : getJoinPartyList(partyId, 1));

            //遍历暂时存在temp的key-value
            for (String key : tempMap.keySet()) {
                page.putField(key, tempMap.get(key));
            }
            //添加报名人链接
        } else if (page.getUrl().regex(ListReg).match()) {
//            page.addTargetRequests(page.getHtml().links().regex(partyListReg).all());
            page.addTargetRequest(page.getHtml().links().regex(partyListReg).get());
//            page.addTargetRequests(page.getHtml().links().regex(ListReg).all());
        }

    }

    @Override
    public Site getSite() {
        return site;
    }


    /**
     * @param url
     * @return partyId
     */
    private  String getPartyId(String url) {
        Matcher matcher = partyIdPattern.matcher(url);
        if (matcher.find()) {
            String s = matcher.group();
            return s.substring(6, s.length() - 5);
        } else {
            return null;
        }
    }

    /**
     * @param author_url
     * @return author_id
     */
    private String getAuthorId(String author_url){
//        http://www.hdb.com/timeline/cam61.html
        Matcher matcher=Pattern.compile("timeline/.*?\\.html").matcher(author_url);
        if(matcher.find()){
            String s=matcher.group();
            return s.substring(9,s.length()-5);
        }else {
            return null;
        }
    }


    private String getAuthorInfo(String authorId,String partyId){
        String url="http://www.hdb.com/query/api:124?user_id=%s&info_id=%s&info_type=party";
        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, authorId,partyId))
                .build();
        Response response = null;
        try {
            response = client.newCall(request).execute();
            String json = response.body().string();
            return json;
        } catch (IOException e) {
            return null;
        }
    }
    /**
     * 获取浏览和分享的jsonStr
     *
     * @param partyId
     * @return "{‘share’:123,‘hits’:1}"
     */
    private String getHitAndShareString(String partyId) {
        String url = "http://www.hdb.com/query/api:51?info_type=party&info_id=%s";
        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, partyId))
                .build();
        Response response = null;
        try {
            response = client.newCall(request).execute();
            String json = response.body().string();
            return json;
        } catch (IOException e) {
            return "{\"share\":null,\"hits\":null}";
        }

    }

    private JSONArray getJoinPartyList(String partyId, int pageIndex) {

        String url = "http://www.hdb.com/query/api:53?party_id=%s&page_num=%s";

        okhttp3.Request request = new okhttp3.Request.Builder()
                .url(String.format(url, partyId, pageIndex))
                .build();
        Response response;
        String json;
        try {
            response = client.newCall(request).execute();
            json = response.body().string();

        } catch (IOException e) {
            return new JSONArray();
//            e.printStackTrace();
//            return
        }

        JSONObject jsonObj = JSON.parseObject(json);
        //报名参加人数
        if (pageIndex == 1) {
            tempMap.put("join_total_num", jsonObj.getInteger("join_total_num"));
        }
        if (jsonObj.containsKey("join_party_list")) {
            JSONArray JoinList = jsonObj.getJSONArray("join_party_list");
            if (jsonObj.getString("has_next_page").equals("1")) {
                JSONArray nextJsonArray = getJoinPartyList(partyId, ++pageIndex);
                JoinList.addAll(nextJsonArray);
            }
            return JoinList;
        } else {
            return new JSONArray();
        }
    }


    public static void main(String[] args) {
        Spider.create(new SpiderProcessor("xiamen")).addUrl("http://www.hdb.com/xiamen/0-0-2-0-1/").addPipeline(new MysqlPipeline()).thread(5).run();
    }


}
